#ifndef __PAGELIBPRE_H__
#define __PAGELIBPRE_H__

#include <vector>
#include <map>
#include <set>
#include <unordered_map>
#include "cppjieba/Jieba.hpp"
#include "webPage.h"
#include "splitTool.h"
using namespace std;

class PageLibPreprocessor
{
public:
    void createWebtxt();
    void createAllwordToDoc();
    void creatPageVec();
    void buildInvertIndexMap();

    void storeInvertIndex();
    void storeOnDisk();
    void getWordWeightSumInWebPage();   //获取每篇网页中所有词语的权重之和
private:
    vector<WebPage> _pageList;//网页库
    unordered_map<string,map<int,double>> _invertIndex;//倒排索引库
    vector<double> _WordWeightSumInWebPage; //每篇网页中所有词语的权重之和

    unordered_map<string,set<int>> _wordinHowManyDoc;
};

#endif